import re,os,random,shlex,subprocess

class data:
    def __init__(self,trainingFile):
        self.fileName=trainingFile
        self.posSam=[]
        self.negSam=[]
        for ctr,line in enumerate(open(trainingFile)):
            if (re.match(r'\+1',line)):
                self.posSam.append(line)
            else:
                self.negSam.append(line)

class randomSampler:
    def __init__(self,noTrain,data,trainDir):
        self.noTrain=noTrain
        self.posSam=data.posSam
        self.negSam=data.negSam
        self.trainDir=trainDir
        os.chdir(self.trainDir)
        self.samples=[]
        for i in xrange(0,noTrain):
            self.samples.append(open(os.path.splitext(os.path.basename(data.fileName))[0]+'_'+str(i+1)+'.train','w'))
        

    def sample(self):
        for i in xrange(0,self.noTrain):
            print 'the count of posSam is '+str(len(self.posSam))
            print 'writing to file '+self.samples[i].name+' in the directory '+os.getcwd()
            for ctr,item in enumerate(self.posSam):
                self.samples[i].write(item)
                self.samples[i].write(self.negSam[random.randint(0,len(self.posSam)-1)])
                self.samples[i].write(self.negSam[random.randint(0,len(self.posSam)-1)])
            self.samples[i].close()

class ProcessManager:
    def __init__(self,cmd,noConProcess,fileBase,trainDir,logBase):
        self.cmd=cmd
        self.noConProcess=noConProcess
        self.fileBase=fileBase
        self.trainDir=trainDir
        self.logBase=logBase

    def  prepareTrainProcList(self,noofFiles):
        self.processList=[]
        for ctr in xrange(0,noofFiles):
            self.processList.append(shlex.split(self.cmd+' '+self.fileBase+'_'+str(ctr+1)+'.train'+' '+self.fileBase+'_'+str(ctr+1)+'.model'))

    def  prepareClassifyProcList(self,noofFiles):
        self.processList=[]
        for ctr in xrange(0,noofFiles):
            self.processList.append(shlex.split(self.cmd+' valid.test '+self.fileBase+'_'+str(ctr+1)+'.model'+' '+self.fileBase+'_'+str(ctr+1)+'.out'))
      
    def LaunchandManage(self):
        os.chdir(self.trainDir)
        self.processes=[]
        maxctr=0
        for ictr in xrange(0,self.noConProcess):
            self.processes.append(subprocess.Popen(self.processList[ictr],stderr=subprocess.STDOUT,stdout=open(self.logBase+str(ictr)+'_log','w')))
            maxCtr=ictr
        while(1):
            for i in range(0,self.noConProcess):
                if (self.processes[i].poll()!=None):
                    if maxCtr<len(self.processList)-1:
                        self.processes[i]=subprocess.Popen(self.processList[maxCtr],stderr=subprocess.STDOUT,stdout=open(self.logBase+str(maxCtr)+'_log','w'))
                        maxCtr+=1
                    else:
                        return



def createSamples():
    trainingData=data('/root/hearst/open_flag.train')
    sampler=randomSampler(8,trainingData,'/root/hearst/randomensemble')
    sampler.sample()

def buildEnsembleModels():
    pm=ProcessManager('/root/svmlight/svm_learn -m 1024 -b 0 -j 2 -e 0.0001',8,'open_flag','/root/hearst/randomensemble','svm_learn')
    pm.prepareTrainProcList(8)
    pm.LaunchandManage()
    del pm
    classifyPm=ProcessManager('/root/svmlight/svm_classify',8,'open_flag','/root/hearst/randomensemble','svm_classify')
    classifyPm.prepareClassifyProcList(8)
    classifyPm.LaunchandManage()
            
            
def createClickSamples():
    trainingData=data('/root/hearst/click_flag.train')
    sampler=randomSampler(8,trainingData,'/root/hearst/randomensemble')
    sampler.sample()

def buildClickEnsembleModels():
    pm=ProcessManager('/root/svmlight/svm_learn -m 1024 -b 0 -j 2 -e 0.0001',8,'click_flag','/root/hearst/randomensemble','click_svm_learn')
    pm.prepareTrainProcList(8)
    pm.LaunchandManage()
    del pm
    classifyPm=ProcessManager('/root/svmlight/svm_classify',8,'click_flag','/root/hearst/randomensemble','click_svm_classify')
    classifyPm.prepareClassifyProcList(8)
    classifyPm.LaunchandManage()
    
def consolidateResults():
    os.chdir('/root/hearst/randomensemble')
    openResultFiles=[]
    clickResultFiles=[]
    finalRes=open('final.res','w')
    
    for i in xrange(1,9):
        openResultFiles.append(open('open_flag_'+str(i)+'.out','r'))
        clickResultFiles.append(open('click_flag_'+str(i)+'.out','r'))


    for line in open('valid_id.test','r'):
        open_pred=0
        click_pred=0
        open_flg=[]
        click_flg=[]
        for i in xrange(0,len(openResultFiles)):
            open_flg.append(float(openResultFiles[i].readline()))
            click_flg.append(float(clickResultFiles[i].readline()))
        for i in xrange(0,len(open_flg)):
            if open_flg[i]>0:
                open_flg[i]=1
            else:
                open_flg[i]=-1
        for i in xrange(0,len(click_flg)):
            if click_flg[i]>0:
                click_flg[i]=1
            else:
                click_flg[i]=-1
        if sum(open_flg)>=1:
            open_pred=1
        if sum(open_flg)>=1 and sum(click_flg)>=1:
            click_pred=1
        finalRes.write(line.rstrip('\n')+','+str(open_pred)+','+str(click_pred)+'\n')
            
def performAll():
   createSamples()
   buildEnsembleModels()
   createClickSamples()
   buildClickEnsembleModels()
   consolidateResults()            
        
    
